#!/usr/bin/perl -I/opt/down
use kosher;
use strict;

chdir '/opt/down/ubuntu/dists';
our $base='http://archive.ubuntu.com/ubuntu/dists';
my ($id,$gid)=(getpwnam'sky')[2,3];

use LWP;
use LWP::ConnCache;

my $s=time;
my $ru = LWP::UserAgent->new;
$ru->agent("Gimly Kosher Nightly Fetcher/1.0");
$ru->conn_cache(LWP::ConnCache->new());
my $all=1;
my $fake=0;

if (-t STDIN)
{
    if ($ARGV[0] eq 3){$all=2}
    if ($ARGV[0] eq 4){$all=4}
    elsif ($ARGV[0] eq 2){$fake=1}
    elsif (!defined($ARGV[0])){$all=0}
}

my ($total,$cnt)=(0,0);
sub Mirror
{
    my $file=$_[0];
    my $url="$base/$file";
    
    my ($sz,$res)=(0);
    if ($fake) {print $file, 'FAKE'} else
    {
    $res=$ru->mirror($url,$file);
    $sz=${$$res{'_headers'}}{'content-length'};
    
    print $file,$$res{'_rc'},$$res{'_msg'},$sz;#%$res,%{${$res}{'_headers'}};
    chown($id,$gid,$file),
    $total+=$sz,$cnt++ if $sz;
    }
    $sz;
}

$\="\n",$,="\t";
my $rel='Release';
my $relg=$rel.'.gpg';
if ($all!=4){
foreach my $release (qw/gutsy hardy/)
{
foreach my $dirtail ('',qw/-security -updates -proposed/)
{
    my $dir=$release.$dirtail;
    next if $dir eq 'gutsy-proposed'; # no needs
#    next unless -d $dir;
    my ($res,$sz,$file,$has,$ptr);
    mkdir($dir,0775),chown($id,$gid,$dir) unless -d $dir ;
    
    $has=1;#force reload
    foreach $file ($relg,$rel)
    {
    last unless Mirror("$dir/$file");
    $has=1;
    }
    next unless $has;
    open IN,"$dir/$rel";
    my %best=();
    while (<IN>)
    {
#	print "Try: $_";
    	next unless /^ (\w{32})\s+(\d+) (\S+)/;
	my ($md0,$s0,$name)=($1,$2,$3);
	next unless $name =~ m{^(\w+)/((?:|debian-installer/)binary-i386|source)/(Packages|Sources|Release)(\.gz|\.bz2|)$};
	next if exists $best{"$1/$2/$3.bz2"};
	next if $4 eq '' && exists $best{"$1/$2/$3.gz"};
	
	delete $best{"$1/$2/$3.gz"} if $4 eq '.bz2';
	delete $best{"$1/$2/$3"}  if $4 eq '.gz' || $4 eq '.bz2';
	
	$best{"$1/$2/$3$4"}=[$md0,$s0];		
    } close IN;
    
    foreach my $name (sort keys %best)
    {
	my ($md0,$s0)=@{$best{$name}};
	
	my @path=split /\//,$name;
	my $fname=pop @path;

	$file=$dir;
	
foreach (@path)
{
$file.='/'.$_;
next if -d $file;
die "$file: $!" unless mkdir($file, 0775);
chown($id,$gid,$file);
}
#$file.='/'.$fname;
	$file="$dir/$name";
#	my $s1=(stat$in)[7];
	Mirror($file);

	next unless $name =~ m'^(\w+)/binary-i386/Packages\.bz2$' && (!$dirtail || $dirtail eq '-proposed');
	$file="$dir/$1/i18n";
	mkdir($file,0775),chown($id,$gid,$file) unless -d $file;
	$file="$file/Translation-ru.bz2";
	
	Mirror($file);
    }
}
#last unless $all;
}
} # all!=4
chdir '..';

if ($all!=4)
{
print `./check.pl`;
#wget -N /gutsy{,-security,-updates}/Release{.gpg,}
`chown -R sky:sky dists` unless $<;
}

sub Mirror2
{
    my ($url,$file)=@_;
    $url=~ /\/([^\/]+)$/,$file=$1 unless $file;
#    open(OUT,'>'.$file),close(OUT),utime(0,0,$file) unless -f $file;
    my $res=$ru->mirror($url,$file);
		
    my $sz=${$$res{'_headers'}}{'content-length'};
    print $file,$$res{'_rc'},$$res{'_msg'},$sz;#%$res,%{${$res}{'_headers'}};
    chown($id,$gid,$file),
    $total+=$sz,$cnt++ if $sz;
    $sz;
}
#exit;
chdir '..';

if (($all==1 || $all==4) && !$fake)
{

for my $dir (qw/WU/)
{
chdir $dir;
open IN,'urls.lst';
print "Checking: $dir";
while (<IN>)
{
    chomp;
    my ($url,$file)=split/\#/,$_,2;
    unless($file)
    {
	$file = $&.$' if $url=~ /Current/;
    }
        
    next if $all==4 && $url!~ /commonupdater/;
    
    next unless $url && $url =~ /^http:/;
    Mirror2($url,$file);
    if ($file =~ /gdeltaavv.ini$/)
    {
	FetchInc($url,$file);
    }
}
chdir '..';
}
}

sub FetchInc($$)
{
    my ($url,$file)=@_;
    open IN,$file or return;
    undef local $/;$_=<IN>;close IN;
    $file=~ s{[^/]*$}{};
    $url =~ s{[^/]*$}{};
    return unless /CurrentVersion=(\d+)/;
    my $ver=$1;
    my $from=5307;
    while ($from<$ver)
    {
	my $id=$from++.$from;
	my $f=$id.'.upd';
#	print("Mirror ",$url.$f,' to ',$file.$f)
	Mirror2($url.$f ,$file.$f) 
	unless -f $file.$f;
	$f=$id.'avv.gem';
#	print("Mirror ",$url.$f,' to ',$file.$f)
	Mirror2($url.$f ,$file.$f)
	unless -f $file.$f;
    }
    
}
$s=time-$s;
$s=sprintf"%u:%02u",int($s/60),$s % 60;

print "Downloaded: $cnt files of $total bytes in $s\n";
